package org.myorg;
	
	import java.io.BufferedReader;
import java.io.FileReader;
import java.io.IOException;
import java.text.Normalizer;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
import java.util.StringTokenizer;
import java.util.regex.Pattern;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.filecache.DistributedCache;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.FileInputFormat;
import org.apache.hadoop.mapred.FileOutputFormat;
import org.apache.hadoop.mapred.JobClient;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.MapReduceBase;
import org.apache.hadoop.mapred.Mapper;
import org.apache.hadoop.mapred.OutputCollector;
import org.apache.hadoop.mapred.Reducer;
import org.apache.hadoop.mapred.Reporter;
import org.apache.hadoop.mapred.TextInputFormat;
import org.apache.hadoop.mapred.TextOutputFormat;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
	
	public class WordCount extends Configured implements Tool {
	
	    public static class Map extends MapReduceBase implements Mapper<LongWritable, Text, Text, IntWritable> {
	
	      static enum Counters { INPUT_WORDS }
	
	      private final static IntWritable one = new IntWritable(1);
	      private Text word = new Text();
	      private Text text = new Text();
	
	      private boolean caseSensitive = true;
	      private Set<String> patternsToSkip = new HashSet<String>();

	      private long numRecords = 0;
	      private String inputFile;

	      public void configure(JobConf job) {
	        caseSensitive = job.getBoolean("wordcount.case.sensitive", true);
	        inputFile = job.get("map.input.file");
	
	        if (job.getBoolean("wordcount.skip.patterns", false)) {
	          Path[] patternsFiles = new Path[0];
	          try {
	            patternsFiles = DistributedCache.getLocalCacheFiles(job);
	          } catch (IOException ioe) {
	            System.err.println("Caught exception while getting cached files: " + StringUtils.stringifyException(ioe));
	          }
	          for (Path patternsFile : patternsFiles) {
	            parseSkipFile(patternsFile);
	          }
	        }
	      }
	      public static String Filter(String str)
	      {
	      
	      StringTokenizer st = new StringTokenizer(str);
	        String stopwords = "a"+
	                  "able"+
	                  "about"+
	                  "above"+
	                  "according"+
	                  "accordingly"+
	                  "across"+
	                  "actually"+
	                  "after"+
	                  "afterwards"+
	                  "again"+
	                  "against"+
	                  "all"+
	                  "allow"+
	                  "allows"+
	                  "almost"+
	                  "alone"+
	                  "along"+
	                  "already"+
	                  "also"+
	                  "although"+
	                  "always"+
	                  "am"+
	                  "among"+
	                  "amongst"+
	                  "an"+
	                  "and"+
	                  "another"+
	                  "any"+
	                  "anybody"+
	                  "anyhow"+
	                  "anyone"+
	                  "anything"+
	                  "anyway"+
	                  "anyways"+
	                  "anywhere"+
	                  "apart"+
	                  "appear"+
	                  "appreciate"+
	                  "appropriate"+
	                  "are"+
	                  "around"+
	                  "as"+
	                  "aside"+
	                  "ask"+
	                  "asking"+
	                  "associated"+
	                  "at"+
	                  "available"+
	                  "away"+
	                  "awfully"+
	                  "b"+
	                  "be"+
	                  "became"+
	                  "because"+
	                  "become"+
	                  "becomes"+
	                  "becoming"+
	                  "been"+
	                  "before"+
	                  "beforehand"+
	                  "behind"+
	                  "being"+
	                  "believe"+
	                  "below"+
	                  "beside"+
	                  "besides"+
	                  "best"+
	                  "better"+
	                  "between"+
	                  "beyond"+
	                  "both"+
	                  "brief"+
	                  "but"+
	                  "by"+
	                  "c"+
	                  "came"+
	                  "can"+
	                  "cannot"+
	                  "cant"+
	                  "cause"+
	                  "causes"+
	                  "certain"+
	                  "certainly"+
	                  "changes"+
	                  "clearly"+
	                  "co"+
	                  "com"+
	                  "come"+
	                  "comes"+
	                  "concerning"+
	                  "consequently"+
	                  "consider"+
	                  "considering"+
	                  "contain"+
	                  "containing"+
	                  "contains"+
	                  "corresponding"+
	                  "could"+
	                  "course"+
	                  "currently"+
	                  "d"+
	                  "definitely"+
	                  "described"+
	                  "despite"+
	                  "did"+
	                  "different"+
	                  "do"+
	                  "does"+
	                  "doing"+
	                  "done"+
	                  "down"+
	                  "downwards"+
	                  "during"+
	                  "e"+
	                  "each"+
	                  "edu"+
	                  "eg"+
	                  "eight"+
	                  "either"+
	                  "else"+
	                  "elsewhere"+
	                  "enough"+
	                  "entirely"+
	                  "especially"+
	                  "et"+
	                  "etc"+
	                  "even"+
	                  "ever"+
	                  "every"+
	                  "everybody"+
	                  "everyone"+
	                  "everything"+
	                  "everywhere"+
	                  "ex"+
	                  "exactly"+
	                  "example"+
	                  "except"+
	                  "f"+
	                  "far"+
	                  "few"+
	                  "fifth"+
	                  "first"+
	                  "five"+
	                  "followed"+
	                  "following"+
	                  "follows"+
	                  "for"+
	                  "former"+
	                  "formerly"+
	                  "forth"+
	                  "four"+
	                  "from"+
	                  "further"+
	                  "furthermore"+
	                  "g"+
	                  "get"+
	                  "gets"+
	                  "getting"+
	                  "given"+
	                  "gives"+
	                  "go"+
	                  "goes"+
	                  "going"+
	                  "gone"+
	                  "got"+
	                  "gotten"+
	                  "greetings"+
	                  "h"+
	                  "had"+
	                  "happens"+
	                  "hardly"+
	                  "has"+
	                  "have"+
	                  "having"+
	                  "he"+
	                  "hello"+
	                  "help"+
	                  "hence"+
	                  "her"+
	                  "here"+
	                  "hereafter"+
	                  "hereby"+
	                  "herein"+
	                  "hereupon"+
	                  "hers"+
	                  "herself"+
	                  "hi"+
	                  "him"+
	                  "himself"+
	                  "his"+
	                  "hither"+
	                  "hopefully"+
	                  "how"+
	                  "howbeit"+
	                  "however"+
	                  "i"+
	                  "ie"+
	                  "if"+
	                  "ignored"+
	                  "immediate"+
	                  "in"+
	                  "inasmuch"+
	                  "inc"+
	                  "indeed"+
	                  "indicate"+
	                  "indicated"+
	                  "indicates"+
	                  "inner"+
	                  "insofar"+
	                  "instead"+
	                  "into"+
	                  "inward"+
	                  "is"+
	                  "it"+
	                  "its"+
	                  "itself"+
	                  "j"+
	                  "just"+
	                  "k"+
	                  "keep"+
	                  "keeps"+
	                  "kept"+
	                  "know"+
	                  "knows"+
	                  "known"+
	                  "l"+
	                  "last"+
	                  "lately"+
	                  "later"+
	                  "latter"+
	                  "latterly"+
	                  "least"+
	                  "less"+
	                  "lest"+
	                  "let"+
	                  "like"+
	                  "liked"+
	                  "likely"+
	                  "little"+
	                  "ll"+ //ed to avoid words like you'll+I'll etc.
	                  "look"+
	                  "looking"+
	                  "looks"+
	                  "ltd"+
	                  "m"+
	                  "mainly"+
	                  "many"+
	                  "may"+
	                  "maybe"+
	                  "me"+
	                  "mean"+
	                  "meanwhile"+
	                  "merely"+
	                  "might"+
	                  "more"+
	                  "moreover"+
	                  "most"+
	                  "mostly"+
	                  "much"+
	                  "must"+
	                  "my"+
	                  "myself"+
	                  "n"+
	                  "name"+
	                  "namely"+
	                  "nd"+
	                  "near"+
	                  "nearly"+
	                  "necessary"+
	                  "need"+
	                  "needs"+
	                  "neither"+
	                  "never"+
	                  "nevertheless"+
	                  "new"+
	                  "next"+
	                  "nine"+
	                  "no"+
	                  "nobody"+
	                  "non"+
	                  "none"+
	                  "noone"+
	                  "nor"+
	                  "normally"+
	                  "not"+
	                  "nothing"+
	                  "novel"+
	                  "now"+
	                  "nowhere"+
	                  "o"+
	                  "obviously"+
	                  "of"+
	                  "off"+
	                  "often"+
	                  "oh"+
	                  "ok"+
	                  "okay"+
	                  "old"+
	                  "on"+
	                  "once"+
	                  "one"+
	                  "ones"+
	                  "only"+
	                  "onto"+
	                  "or"+
	                  "other"+
	                  "others"+
	                  "otherwise"+
	                  "ought"+
	                  "our"+
	                  "ours"+
	                  "ourselves"+
	                  "out"+
	                  "outside"+
	                  "over"+
	                  "overall"+
	                  "own"+
	                  "p"+
	                  "particular"+
	                  "particularly"+
	                  "per"+
	                  "perhaps"+
	                  "placed"+
	                  "please"+
	                  "plus"+
	                  "possible"+
	                  "presumably"+
	                  "probably"+
	                  "provides"+
	                  "q"+
	                  "que"+
	                  "quite"+
	                  "qv"+
	                  "r"+
	                  "rather"+
	                  "rd"+
	                  "re"+
	                  "really"+
	                  "reasonably"+
	                  "regarding"+
	                  "regardless"+
	                  "regards"+
	                  "relatively"+
	                  "respectively"+
	                  "right"+
	                  "s"+
	                  "said"+
	                  "same"+
	                  "saw"+
	                  "say"+
	                  "saying"+
	                  "says"+
	                  "second"+
	                  "secondly"+
	                  "see"+
	                  "seeing"+
	                  "seem"+
	                  "seemed"+
	                  "seeming"+
	                  "seems"+
	                  "seen"+
	                  "self"+
	                  "selves"+
	                  "sensible"+
	                  "sent"+
	                  "serious"+
	                  "seriously"+
	                  "seven"+
	                  "several"+
	                  "shall"+
	                  "she"+
	                  "should"+
	                  "since"+
	                  "six"+
	                  "so"+
	                  "some"+
	                  "somebody"+
	                  "somehow"+
	                  "someone"+
	                  "something"+
	                  "sometime"+
	                  "sometimes"+
	                  "somewhat"+
	                  "somewhere"+
	                  "soon"+
	                  "sorry"+
	                  "specified"+
	                  "specify"+
	                  "specifying"+
	                  "still"+
	                  "sub"+
	                  "such"+
	                  "sup"+
	                  "sure"+
	                  "t"+
	                  "take"+
	                  "taken"+
	                  "tell"+
	                  "tends"+
	                  "th"+
	                  "than"+
	                  "thank"+
	                  "thanks"+
	                  "thanx"+
	                  "that"+
	                  "thats"+
	                  "the"+
	                  "their"+
	                  "theirs"+
	                  "them"+
	                  "themselves"+
	                  "then"+
	                  "thence"+
	                  "there"+
	                  "thereafter"+
	                  "thereby"+
	                  "therefore"+
	                  "therein"+
	                  "theres"+
	                  "thereupon"+
	                  "these"+
	                  "they"+
	                  "think"+
	                  "third"+
	                  "this"+
	                  "thorough"+
	                  "thoroughly"+
	                  "those"+
	                  "though"+
	                  "three"+
	                  "through"+
	                  "throughout"+
	                  "thru"+
	                  "thus"+
	                  "to"+
	                  "together"+
	                  "too"+
	                  "took"+
	                  "toward"+
	                  "towards"+
	                  "tried"+
	                  "tries"+
	                  "truly"+
	                  "try"+
	                  "trying"+
	                  "twice"+
	                  "two"+
	                  "u"+
	                  "un"+
	                  "under"+
	                  "unfortunately"+
	                  "unless"+
	                  "unlikely"+
	                  "until"+
	                  "unto"+
	                  "up"+
	                  "upon"+
	                  "us"+
	                  "use"+
	                  "used"+
	                  "useful"+
	                  "uses"+
	                  "using"+
	                  "usually"+
	                  "uucp"+
	                  "v"+
	                  "value"+
	                  "various"+
	                  "ve"+ //ed to avoid words like I've+you've etc.
	                  "very"+
	                  "via"+
	                  "viz"+
	                  "vs"+
	                  "w"+
	                  "want"+
	                  "wants"+
	                  "was"+
	                  "way"+
	                  "we"+
	                  "welcome"+
	                  "well"+
	                  "went"+
	                  "were"+
	                  "what"+
	                  "whatever"+
	                  "when"+
	                  "whence"+
	                  "whenever"+
	                  "where"+
	                  "whereafter"+
	                  "whereas"+
	                  "whereby"+
	                  "wherein"+
	                  "whereupon"+
	                  "wherever"+
	                  "whether"+
	                  "which"+
	                  "while"+
	                  "whither"+
	                  "who"+
	                  "whoever"+
	                  "whole"+
	                  "whom"+
	                  "whose"+
	                  "why"+
	                  "will"+
	                  "willing"+
	                  "wish"+
	                  "with"+
	                  "within"+
	                  "without"+
	                  "wonder"+
	                  "would"+
	                  "x"+
	                  "y"+
	                  "yes"+
	                  "yet"+
	                  "you"+
	                  "your"+
	                  "yours"+
	                  "yourself"+
	                  "yourselves"+
	                  "z"+
	                  "zero"+"\"";

	       // String stp = "Law"+ "Harvard";
	       String output = null;
	       while(st.hasMoreTokens())
	       {//  System.out.println(st.nextToken());
	       
	        String word =st.nextToken();
	        //System.out.println(word);
	        if(!(stopwords.contains(word)))
	        {   if(output==null){output=word;}
	            else {output=output+" "+word;}
	        }
	       }
	       //System.out.println(output);
	         
	         return output;
	     
	      //    System.out.println(System.getProperty("user.dir"));
	      }
	      public static String unAccent(String s) 
	      {
	        //5
	        //   use sun.text.Normalizer.normalize(s, Normalizer.DECOMP, 0);
	        //
	        String temp = Normalizer.normalize(s, Normalizer.Form.NFD);
	        Pattern pattern = Pattern.compile("\\p{InCombiningDiacriticalMarks}+");
	        return pattern.matcher(temp).replaceAll("");
	    }
	      public static String stemmer(String args)
	      {
	         char[] w = new char[500000];
	         //char ch;
	         String temp = new String(args);
	         //String temp = new String(args);
	         temp = temp.trim();
	         
	         System.out.println(temp.length());
	         System.out.println("In Stemmer");
	         String output="";
	         	
	         Stemmer s = new Stemmer();
	         
	   		

	                //int ch = in.read();
	         int t=0;
	   					
	   			while(true)
	   			{		if(t==temp.length())break;
	           	       int ch =(int) temp.charAt(t);
	           	       t++;
	           	        //System.out.println("ch is:"+ch);
	                 if (Character.isLetter((char) ch))
	                 {
	                    int j = 0;
	                    while(true)
	                    {  ch = Character.toLowerCase((char) ch);
	                       w[j] = (char) ch;
	                       if (j < 500000) j++;
	                       
	                      // if(t<temp.length()-1)
	                        //ch = temp.charAt(t++);
	                       if(t==temp.length()) {ch = -1;}
	                       if(t<=(temp.length()-1))
	                       {
	                       	//if(t==(temp.length()-1)){ ch = -1; }
	                       	ch = (int) temp.charAt(t);
	                       	t++;
	                       }
	                       //System.out.println("ch2 is:"+ch);
	                       if (!Character.isLetter((char) ch))
	                       {
	                          /* to test add(char ch) */
	                          for (int c = 0; c < j; c++) s.add(w[c]);

	                          /* or, to test add(char[] w, int j) */
	                          /* s.add(w, j); */

	                          s.stem();
	                          {  String u;

	                             /* and now, to test toString() : */
	                             u = s.toString();

	                             /* to test getResultBuffer(), getResultLength() : */
	                             /* u = new String(s.getResultBuffer(), 0, s.getResultLength()); */

	                             System.out.print(u);
	                             System.out.print("Before Concatenating"); 
	                             output = output.concat(" ").concat(u);
	                             
	                          }
	                          break;
	                       }
	                    }
	                 
	                 }
	                 if (ch < 0) break;
	                 System.out.print((char)ch);
	              }
	   			return output;
	      }
	
	      private void parseSkipFile(Path patternsFile) {
	        try {
	          BufferedReader fis = new BufferedReader(new FileReader(patternsFile.toString()));
	          String pattern = null;
	          while ((pattern = fis.readLine()) != null) {
	            patternsToSkip.add(pattern);
	          }
	        } catch (IOException ioe) {
	          System.err.println("Caught exception while parsing the cached file '" + patternsFile + "' : " + StringUtils.stringifyException(ioe));
	        }
	      }
	
	      public void map(LongWritable key, Text value, OutputCollector<Text, IntWritable> output, Reporter reporter) throws IOException {
	       /* String line = (caseSensitive) ? value.toString() : value.toString().toLowerCase();
	
	        for (String pattern : patternsToSkip) {
	          line = line.replaceAll(pattern, "");
	        }
	
	        StringTokenizer tokenizer = new StringTokenizer(line);
	        while (tokenizer.hasMoreTokens()) {
	          word.set(tokenizer.nextToken());
	          output.collect(word, one);
	          reporter.incrCounter(Counters.INPUT_WORDS, 1);
	        } */
	    	  String line = value.toString();
	    	  //System.out.println(""+line);
	    	  String str = line;
	    	  String url="";
	    	  String data="";
	    	  int index=str.indexOf("http");
	    	  //String newline = System.getProperty("line.separator");
	    	  
	    	  
	    	  int index2=str.indexOf("ParseText::"); 
	    	  
	    	  //str.
				if(index!=-1)
				{
					url=str.substring(index);
					//System.out.println(url);
					//str = br.readLine();
					//System.out.println(str);
				}
					//String temp;
					//int index2 = str.indexOf("ParseText");
					System.out.println(""+index2);
					if(index2!=-1)
					{
						System.out.println(url);
						data = str.substring(index2);
						//System.out.println(""+data);
						data = unAccent(data);
						data = Filter(data);
						data = data.trim();
						data = stemmer(data);
						StringBuilder valuebuilder = new StringBuilder();	
						StringTokenizer tokenizer = new StringTokenizer(data);
						while (tokenizer.hasMoreTokens())
						{
						//word.set(tokenizer.nextToken());
						valuebuilder.append(tokenizer.nextToken());
						valuebuilder.append("@");
						valuebuilder.append(url);
						this.word.set(valuebuilder.toString());
						output.collect(word, one);
						valuebuilder.setLength(0);
						
						
					}
					
					//reporter.incrCounter(Counters.INPUT_WORDS, 1);
				} 
						
				//System.out.println(url);
				//System.out.println(data);
				
				//word.set(url);
				//text.set(data);
				//output.collect(word,text );
	
	        if ((++numRecords % 100) == 0) {
	          reporter.setStatus("Finished processing " + numRecords + " records " + "from the input file: " + inputFile);
	        }
	      }
	    }
	
	    public static class Reduce extends MapReduceBase implements Reducer<Text, IntWritable, Text, IntWritable> {
	      public void reduce(Text key, Iterator<IntWritable> values, OutputCollector<Text, IntWritable> output, Reporter reporter) throws IOException {
	        /*int sum = 0;
	        while (values.hasNext()) {
	          sum += values.next().get();
	        }
	        output.collect(key, new IntWritable(sum)); */
	    	  //System.out.println(values);
	    	/* String data2;
	    	  Text data = new Text();
	    	  if(values.hasNext())
	    	  {
	    		   data = values.next();
	    		   System.out.println(data);
	    		   //data2.set(data);
	 	    	 
	    	  }
	    	  output.collect(key,data);
	    	  */
	      }
	    }
	
	    public int run(String[] args) throws Exception {
	      JobConf conf = new JobConf(getConf(), WordCount.class);
	      conf.setJobName("wordcount");
	      
	
	      conf.setOutputKeyClass(Text.class);
	      conf.setOutputValueClass(IntWritable.class);
	
	      conf.setMapperClass(Map.class);
	      conf.setCombinerClass(Reduce.class);
	      conf.setReducerClass(Reduce.class);
	      conf.setNumReduceTasks(0);
	
	      conf.setInputFormat(TextInputFormat.class);
	      conf.setOutputFormat(TextOutputFormat.class);
	
	      List<String> other_args = new ArrayList<String>();
	      for (int i=0; i < args.length; ++i) {
	        if ("-skip".equals(args[i])) {
	          DistributedCache.addCacheFile(new Path(args[++i]).toUri(), conf);
	          conf.setBoolean("wordcount.skip.patterns", true);
	        } else {
	          other_args.add(args[i]);
	        }
	      }
	
	      FileInputFormat.setInputPaths(conf, new Path("/Users/ajayreddyanireddy/Desktop/parseFile.txt"));
	      FileOutputFormat.setOutputPath(conf, new Path("/Users/ajayreddyanireddy/Desktop/parseFileMapReduce"));
	
	      JobClient.runJob(conf);
	      return 0;
	    }
	
	    public static void main(String[] args) throws Exception {
	    	 
	      int res = ToolRunner.run(new Configuration(), new WordCount(), args);
	      System.exit(res);
	    }
	}
	



